{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import random"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(array([0.94126699, 0.0176484 , 0.16268795, 0.89367725, 0.89960481,\n",
       "        0.75717816, 0.80557899, 0.05534895, 0.81134799, 0.42980477]),\n",
       " [[1], [1], [1], [1], [1], [1], [1], [1], [1], [1]])"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 虚拟10台老虎机\n",
    "prods = np.random.uniform(size=10)  # 每台老虎机的中奖概率,0-1之间\n",
    "# 记录每台老虎机的返回值\n",
    "rewards = [[1] for _ in prods]\n",
    "prods, rewards"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def choose_one():\n",
    "    \"\"\"\n",
    "    贪婪算法,选择奖励最多的老虎机\n",
    "    \"\"\"\n",
    "    # 得到当前玩了多少次了\n",
    "    play_count = sum([len(i) for i in rewards])\n",
    "    # 随机选择的概率逐渐下降\n",
    "    if random.random() < 1 / play_count:  # 小概率随机选择一台老虎机的拉杆\n",
    "        return random.randint(0, 9)\n",
    "    # 计算每台老虎机的平均奖励\n",
    "    rewards_mean = [np.mean(i) for i in rewards]\n",
    "    # 选择期望奖励最大的那台老虎机\n",
    "    return np.argmax(rewards_mean)\n",
    "\n",
    "\n",
    "choose_one()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[[1, 1], [1], [1], [1], [1], [1], [1], [1], [1], [1]]"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def try_and_play():\n",
    "    \"\"\"\n",
    "    玩老虎机,积累经验\n",
    "    \"\"\"\n",
    "    i = choose_one()\n",
    "    # 玩老虎机,得到结果\n",
    "    reward = 0\n",
    "    if random.random() < prods[i]:\n",
    "        reward = 1\n",
    "    # 记录结果\n",
    "    rewards[i].append(reward)\n",
    "try_and_play()\n",
    "rewards"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(4706.334949991448, 4494)"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 清空一下之前学习的经验\n",
    "rewards = [[1] for _ in prods]\n",
    "\n",
    "\n",
    "def get_result():\n",
    "    # 玩5000次\n",
    "    for _ in range(5000):\n",
    "        try_and_play()\n",
    "    # 最好的结果\n",
    "    target = prods.max() * 5000\n",
    "    # 实际的结果\n",
    "    result = sum([sum(i) for i in rewards])\n",
    "\n",
    "    return target, result\n",
    "\n",
    "\n",
    "get_result()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def choose_one():\n",
    "    \"\"\"\n",
    "    上置信界算法\n",
    "    \"\"\"\n",
    "    # 得到当前玩了多少次了\n",
    "    play_count = sum([len(i) for i in rewards])\n",
    "    play_count = np.array(play_count)\n",
    "    \"\"\"\n",
    "    求出上置信界\n",
    "    分子是总共玩了多少次,取根号后让他的增长速度变慢\n",
    "    分母是每台老虎机玩的次数,乘以2让他的增长速度变快\n",
    "    随着玩的次数增加,分母会很快超过分子的增长速度,导致分数越来越小\n",
    "    具体到每一台老虎机,则是玩的次数越多,分数就越小,也就是ucb的加权越小\n",
    "    所以ucb衡量了每一台老虎机的不确定性,不确定性越大,探索的价值越大\n",
    "    \"\"\"\n",
    "    fz = play_count.sum() ** 0.5\n",
    "    fm = play_count * 2\n",
    "    ucb = fz / fm\n",
    "    # ucb本身取根号 大于1的数会被缩小,小于1的数会被放大,这样保持ucb恒定在一定的数值范围内\n",
    "    ucb = ucb ** 0.5\n",
    "    # 计算每个老虎机的奖励平均\n",
    "    rewards_mean = [np.mean(i) for i in rewards]\n",
    "    rewards_mean = np.array(rewards_mean)\n",
    "    # ucb和期望求和\n",
    "    ucb += rewards_mean\n",
    "    return ucb.argmax()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(4706.334949991448, 4743)"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 清空一下之前学习的经验\n",
    "rewards = [[1] for _ in prods]\n",
    "get_result()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "当数字小的时候,beta分布的概率有很大的随机性\n",
      "0.14873598123909823\n",
      "0.5824124267215814\n",
      "0.15632408899776737\n",
      "0.27512827257222594\n",
      "0.35118089614430076\n",
      "当数字大时,beta分布逐渐稳定\n",
      "0.49739519948584865\n",
      "0.49934500594326686\n",
      "0.5005625890311245\n",
      "0.499732707645106\n",
      "0.4993736483576031\n"
     ]
    }
   ],
   "source": [
    "#beta分布测试\n",
    "print('当数字小的时候,beta分布的概率有很大的随机性')\n",
    "for _ in range(5):\n",
    "    print(np.random.beta(1, 1))\n",
    "\n",
    "print('当数字大时,beta分布逐渐稳定')\n",
    "for _ in range(5):\n",
    "    print(np.random.beta(1e5, 1e5))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "def choose_one():\n",
    "    \"\"\"\n",
    "    汤普森采样算法\n",
    "    \"\"\"\n",
    "\n",
    "    # 求出每个老虎机出1的次数+1\n",
    "    count_1 = [sum(i) + 1 for i in rewards]\n",
    "\n",
    "    # 求出每个老虎机出0的次数+1\n",
    "    count_0 = [sum(1 - np.array(i)) + 1 for i in rewards]\n",
    "\n",
    "    # 按照beta分布计算奖励分布,这可以认为是每一台老虎机中奖的概率\n",
    "    beta = np.random.beta(count_1, count_0)\n",
    "\n",
    "    return beta.argmax()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(4706.334949991448, 4647)"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 清空一下之前学习的经验\n",
    "rewards = [[1] for _ in prods]\n",
    "get_result()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.8.13 ('d2l')",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.13"
  },
  "orig_nbformat": 4,
  "vscode": {
   "interpreter": {
    "hash": "6bdcd3dff61cfecb093f68bfbd67338d4a9739a04d65268bf3b66287b86a0a9e"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
